{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from factories.vgkr_v2.config_v2 import *\n",
    "import mmcv\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "from pandas import Series, DataFrame\n",
    "import h5py\n",
    "import os.path as osp\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_preprocess(phrase):\n",
    "    \"\"\" preprocess a sentence: lowercase, clean up weird chars, remove punctuation \"\"\"\n",
    "    replacements = {\n",
    "      '½': 'half',\n",
    "      '—' : '-',\n",
    "      '™': '',\n",
    "      '¢': 'cent',\n",
    "      'ç': 'c',\n",
    "      'û': 'u',\n",
    "      'é': 'e',\n",
    "      '°': ' degree',\n",
    "      'è': 'e',\n",
    "      '…': '',\n",
    "    }\n",
    "    table1 = str.maketrans(\"\", \"\", '!\\'*+-.,:;?^`{|}~')\n",
    "    #table2 = str.maketrans(\"\", \"\", \"0123456789\")\n",
    "    #phrase = phrase.encode('utf-8')\n",
    "    phrase = phrase.lstrip(' ').rstrip(' ')\n",
    "    #for k, v in replacements.items():\n",
    "    #    phrase = phrase.replace(k, v)\n",
    "        \n",
    "    phrase = str(phrase).lower().translate(table1)\n",
    "    phrase = ' '.join(phrase.split())\n",
    "    return phrase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Re-organize the data: Using the pandas\n",
    "1. With pandas, we write the record of each image into a DataFrame.\n",
    "2. During joint operation between VG and COCO, we find some duplicated images. Before, we think that there are 51,498 overlapped images. However, 290 images are duplicated. So there actually are 51,208 overlapped images. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_meta = mmcv.load(meta_file)\n",
    "corrupted_ims = ['1592.jpg', '1722.jpg', '4616.jpg', '4617.jpg']\n",
    "image_meta_tmp = []\n",
    "for item in image_meta:\n",
    "    if str(item['image_id']) + '.jpg' not in corrupted_ims:\n",
    "        image_meta_tmp.append(item)\n",
    "image_meta = image_meta_tmp\n",
    "# load the train/test flag from annotations\n",
    "roi_h5 = h5py.File(roidb_file, 'r')\n",
    "data_split = roi_h5['split'][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_vgids = [item['url'].split('/')[-1][:-4] for item in image_meta]\n",
    "meta_vgids = [str(item['image_id']) for item in image_meta]\n",
    "meta_cocoids = [str(item['coco_id']) for item in image_meta]\n",
    "meta_flickrids = [str(item['flickr_id']) for item in image_meta]\n",
    "meta_paths = ['/'.join(item['url'].split('/')[-2:]) for item in image_meta]\n",
    "meta_heights = [item['height'] for item in image_meta]\n",
    "meta_widths = [item['width'] for item in image_meta]\n",
    "meta_dbidx = [i for i in range(len(image_meta))]\n",
    "meta_infos = DataFrame({\"meta_dbidx\": meta_dbidx, \"path_vgids\": path_vgids, \"meta_vgids\": meta_vgids, \n",
    "                        \"meta_cocoids\": meta_cocoids, \"meta_flickr_ids\": meta_flickrids, \"meta_paths\": meta_paths, \"meta_heights\": meta_heights,\n",
    "                        \"meta_widths\": meta_widths, \"vg150_split\": data_split})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_dbidx</th>\n",
       "      <th>path_vgids</th>\n",
       "      <th>meta_vgids</th>\n",
       "      <th>meta_cocoids</th>\n",
       "      <th>meta_flickr_ids</th>\n",
       "      <th>meta_paths</th>\n",
       "      <th>meta_heights</th>\n",
       "      <th>meta_widths</th>\n",
       "      <th>vg150_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K_2/1.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/2.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/3.jpg</td>\n",
       "      <td>480</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/4.jpg</td>\n",
       "      <td>480</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/5.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108068</th>\n",
       "      <td>108068</td>\n",
       "      <td>2417992</td>\n",
       "      <td>2417992</td>\n",
       "      <td>41116</td>\n",
       "      <td>9669763691</td>\n",
       "      <td>VG_100K_2/2417992.jpg</td>\n",
       "      <td>372</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108069</th>\n",
       "      <td>108069</td>\n",
       "      <td>2417993</td>\n",
       "      <td>2417993</td>\n",
       "      <td>109761</td>\n",
       "      <td>9680535991</td>\n",
       "      <td>VG_100K_2/2417993.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108070</th>\n",
       "      <td>108070</td>\n",
       "      <td>2417994</td>\n",
       "      <td>2417994</td>\n",
       "      <td>27438</td>\n",
       "      <td>9684712191</td>\n",
       "      <td>VG_100K_2/2417994.jpg</td>\n",
       "      <td>246</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108071</th>\n",
       "      <td>108071</td>\n",
       "      <td>2417995</td>\n",
       "      <td>2417995</td>\n",
       "      <td>None</td>\n",
       "      <td>9721821291</td>\n",
       "      <td>VG_100K_2/2417995.jpg</td>\n",
       "      <td>294</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108072</th>\n",
       "      <td>108072</td>\n",
       "      <td>2417997</td>\n",
       "      <td>2417997</td>\n",
       "      <td>None</td>\n",
       "      <td>9777145091</td>\n",
       "      <td>VG_100K_2/2417997.jpg</td>\n",
       "      <td>332</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108073 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        meta_dbidx path_vgids meta_vgids meta_cocoids meta_flickr_ids  \\\n",
       "0                0          1          1         None            None   \n",
       "1                1          2          2         None            None   \n",
       "2                2          3          3         None            None   \n",
       "3                3          4          4         None            None   \n",
       "4                4          5          5         None            None   \n",
       "...            ...        ...        ...          ...             ...   \n",
       "108068      108068    2417992    2417992        41116      9669763691   \n",
       "108069      108069    2417993    2417993       109761      9680535991   \n",
       "108070      108070    2417994    2417994        27438      9684712191   \n",
       "108071      108071    2417995    2417995         None      9721821291   \n",
       "108072      108072    2417997    2417997         None      9777145091   \n",
       "\n",
       "                   meta_paths  meta_heights  meta_widths  vg150_split  \n",
       "0             VG_100K_2/1.jpg           600          800            0  \n",
       "1               VG_100K/2.jpg           600          800            0  \n",
       "2               VG_100K/3.jpg           480          640            0  \n",
       "3               VG_100K/4.jpg           480          640            0  \n",
       "4               VG_100K/5.jpg           600          800            0  \n",
       "...                       ...           ...          ...          ...  \n",
       "108068  VG_100K_2/2417992.jpg           372          500            2  \n",
       "108069  VG_100K_2/2417993.jpg           500          500            2  \n",
       "108070  VG_100K_2/2417994.jpg           246          500            2  \n",
       "108071  VG_100K_2/2417995.jpg           294          500            2  \n",
       "108072  VG_100K_2/2417997.jpg           332          500            2  \n",
       "\n",
       "[108073 rows x 9 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_infos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Joint: VG and COCO\n",
    "1. During joint operation between VG and COCO, we find some duplicated images. When constructing VGKR_v1 (ECCV version), we think that there are 51,498 overlapped images. However, 290 images are duplicated. So there actually are 51,208 overlapped images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['100000' '100001' '100006' ... '99984' '99995' '99996'] [1 1 1 ... 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "#meta_infos\n",
    "#meta_infos.loc[meta_infos['meta_cocoids'].isin(['140244'])]\n",
    "#meta_infos['meta_cocoids'].isin(['140244'])\n",
    "#meta_infos.loc[meta_infos['meta_cocoids'].isin(['100711'])]['split'].iloc[1]\n",
    "unique, counts = np.unique(list(meta_infos[\"meta_cocoids\"]), return_counts=True)  # the last one is None\n",
    "unique = unique[:-1]\n",
    "counts = counts[:-1]\n",
    "print(unique, counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105 185\n"
     ]
    }
   ],
   "source": [
    "# here, we show the situation: 290 cocoids containing repeatance has two repeated images, 105 pairs are both training split, 185 are test split.  \n",
    "num_repeat_train, num_repeat_train_test = 0, 0\n",
    "for cid in (np.where(counts>1)[0]):\n",
    "    split_slice = meta_infos.loc[meta_infos['meta_cocoids'].isin([unique[cid]])]['vg150_split']\n",
    "    if split_slice.iloc[0] == 0 and split_slice.iloc[1] == 2:\n",
    "        num_repeat_train += 1\n",
    "    else:\n",
    "        num_repeat_train_test += 1\n",
    "print(num_repeat_train, num_repeat_train_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "290\n"
     ]
    }
   ],
   "source": [
    "# i.e, there are 51498-290=51208 unique images that are covered by coco and vg. \n",
    "repeat_cocoids = unique[np.where(counts>1)[0]]\n",
    "print(len(repeat_cocoids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "duplicated_keep_idxes = {}\n",
    "for cid in repeat_cocoids:\n",
    "    dbidx_slice = meta_infos.loc[meta_infos['meta_cocoids'].isin([cid])]['meta_dbidx']\n",
    "    split_slice = meta_infos.loc[meta_infos['meta_cocoids'].isin([cid])]['vg150_split']\n",
    "    assert split_slice.iloc[0] == 0\n",
    "    duplicated_keep_idxes[cid] = dbidx_slice.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "290"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(duplicated_keep_idxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for each duplicated pair, we use the first one as train split, and drop the second one\n",
    "vgcoco_split = []\n",
    "for idx, cid, s in zip(meta_dbidx, meta_cocoids, data_split):\n",
    "    if cid == 'None':\n",
    "        vgcoco_split.append(-1)\n",
    "    elif cid in repeat_cocoids:\n",
    "        if idx == duplicated_keep_idxes[cid]:\n",
    "            vgcoco_split.append(0)\n",
    "        else:\n",
    "            vgcoco_split.append(-1)  # the second one, just drop it\n",
    "    else:\n",
    "        vgcoco_split.append(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35155 16053\n"
     ]
    }
   ],
   "source": [
    "print(len(np.where(np.array(vgcoco_split)==0)[0]), len(np.where(np.array(vgcoco_split)==2)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note: this is the split for VG-COCO: using the COCO ids as a bridge to obtain the overlapped images (51,208), then split them according to the VG150 split. \n",
    "meta_infos['vgcoco_split'] = vgcoco_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_dbidx</th>\n",
       "      <th>path_vgids</th>\n",
       "      <th>meta_vgids</th>\n",
       "      <th>meta_cocoids</th>\n",
       "      <th>meta_flickr_ids</th>\n",
       "      <th>meta_paths</th>\n",
       "      <th>meta_heights</th>\n",
       "      <th>meta_widths</th>\n",
       "      <th>vg150_split</th>\n",
       "      <th>vgcoco_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K_2/1.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/2.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/3.jpg</td>\n",
       "      <td>480</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/4.jpg</td>\n",
       "      <td>480</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/5.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108068</th>\n",
       "      <td>108068</td>\n",
       "      <td>2417992</td>\n",
       "      <td>2417992</td>\n",
       "      <td>41116</td>\n",
       "      <td>9669763691</td>\n",
       "      <td>VG_100K_2/2417992.jpg</td>\n",
       "      <td>372</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108069</th>\n",
       "      <td>108069</td>\n",
       "      <td>2417993</td>\n",
       "      <td>2417993</td>\n",
       "      <td>109761</td>\n",
       "      <td>9680535991</td>\n",
       "      <td>VG_100K_2/2417993.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108070</th>\n",
       "      <td>108070</td>\n",
       "      <td>2417994</td>\n",
       "      <td>2417994</td>\n",
       "      <td>27438</td>\n",
       "      <td>9684712191</td>\n",
       "      <td>VG_100K_2/2417994.jpg</td>\n",
       "      <td>246</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108071</th>\n",
       "      <td>108071</td>\n",
       "      <td>2417995</td>\n",
       "      <td>2417995</td>\n",
       "      <td>None</td>\n",
       "      <td>9721821291</td>\n",
       "      <td>VG_100K_2/2417995.jpg</td>\n",
       "      <td>294</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108072</th>\n",
       "      <td>108072</td>\n",
       "      <td>2417997</td>\n",
       "      <td>2417997</td>\n",
       "      <td>None</td>\n",
       "      <td>9777145091</td>\n",
       "      <td>VG_100K_2/2417997.jpg</td>\n",
       "      <td>332</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108073 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        meta_dbidx path_vgids meta_vgids meta_cocoids meta_flickr_ids  \\\n",
       "0                0          1          1         None            None   \n",
       "1                1          2          2         None            None   \n",
       "2                2          3          3         None            None   \n",
       "3                3          4          4         None            None   \n",
       "4                4          5          5         None            None   \n",
       "...            ...        ...        ...          ...             ...   \n",
       "108068      108068    2417992    2417992        41116      9669763691   \n",
       "108069      108069    2417993    2417993       109761      9680535991   \n",
       "108070      108070    2417994    2417994        27438      9684712191   \n",
       "108071      108071    2417995    2417995         None      9721821291   \n",
       "108072      108072    2417997    2417997         None      9777145091   \n",
       "\n",
       "                   meta_paths  meta_heights  meta_widths  vg150_split  \\\n",
       "0             VG_100K_2/1.jpg           600          800            0   \n",
       "1               VG_100K/2.jpg           600          800            0   \n",
       "2               VG_100K/3.jpg           480          640            0   \n",
       "3               VG_100K/4.jpg           480          640            0   \n",
       "4               VG_100K/5.jpg           600          800            0   \n",
       "...                       ...           ...          ...          ...   \n",
       "108068  VG_100K_2/2417992.jpg           372          500            2   \n",
       "108069  VG_100K_2/2417993.jpg           500          500            2   \n",
       "108070  VG_100K_2/2417994.jpg           246          500            2   \n",
       "108071  VG_100K_2/2417995.jpg           294          500            2   \n",
       "108072  VG_100K_2/2417997.jpg           332          500            2   \n",
       "\n",
       "        vgcoco_split  \n",
       "0                 -1  \n",
       "1                 -1  \n",
       "2                 -1  \n",
       "3                 -1  \n",
       "4                 -1  \n",
       "...              ...  \n",
       "108068             2  \n",
       "108069             2  \n",
       "108070             2  \n",
       "108071            -1  \n",
       "108072            -1  \n",
       "\n",
       "[108073 rows x 10 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_infos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constructing VGKR_v2: grounding the relations from captions to images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "coco_entities = mmcv.load(coco_caption_entities_file)  # 123,169, the key is coco_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not osp.isfile(cap_to_sg_file_v2):\n",
    "    capsg_v1 = mmcv.load(cap_to_sg_file_v1) # generating using the previous idx, so remain 51,498 items. the key is vg_id\n",
    "    captions_vg = mmcv.load(captions_vg_file)   # 51,498 items, the key is vg_id\n",
    "    capsg = dict()\n",
    "    for key, sg_list in capsg_v1.items():\n",
    "        cap_list = captions_vg[key]\n",
    "        new_sg_list = []\n",
    "        for sg, cap in zip(sg_list, cap_list):\n",
    "            sg['caption'] = cap\n",
    "            new_sg_list.append(sg)\n",
    "        capsg[key] = new_sg_list\n",
    "    mmcv.dump(capsg, cap_to_sg_file_v2)\n",
    "else:\n",
    "    capsg = mmcv.load(cap_to_sg_file_v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_dbidx</th>\n",
       "      <th>path_vgids</th>\n",
       "      <th>meta_vgids</th>\n",
       "      <th>meta_cocoids</th>\n",
       "      <th>meta_flickr_ids</th>\n",
       "      <th>meta_paths</th>\n",
       "      <th>meta_heights</th>\n",
       "      <th>meta_widths</th>\n",
       "      <th>vg150_split</th>\n",
       "      <th>vgcoco_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>4996</td>\n",
       "      <td>2415074</td>\n",
       "      <td>2415074</td>\n",
       "      <td>33554</td>\n",
       "      <td>128827281</td>\n",
       "      <td>VG_100K_2/2415074.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>4997</td>\n",
       "      <td>2415075</td>\n",
       "      <td>2415075</td>\n",
       "      <td>290942</td>\n",
       "      <td>163614481</td>\n",
       "      <td>VG_100K_2/2415075.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>4999</td>\n",
       "      <td>2415077</td>\n",
       "      <td>2415077</td>\n",
       "      <td>111842</td>\n",
       "      <td>177587881</td>\n",
       "      <td>VG_100K_2/2415077.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5002</th>\n",
       "      <td>5002</td>\n",
       "      <td>2415080</td>\n",
       "      <td>2415080</td>\n",
       "      <td>100318</td>\n",
       "      <td>393790581</td>\n",
       "      <td>VG_100K_2/2415080.jpg</td>\n",
       "      <td>426</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5004</th>\n",
       "      <td>5004</td>\n",
       "      <td>2415082</td>\n",
       "      <td>2415082</td>\n",
       "      <td>127751</td>\n",
       "      <td>452973481</td>\n",
       "      <td>VG_100K_2/2415082.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108064</th>\n",
       "      <td>108064</td>\n",
       "      <td>2417988</td>\n",
       "      <td>2417988</td>\n",
       "      <td>15751</td>\n",
       "      <td>9419515091</td>\n",
       "      <td>VG_100K_2/2417988.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108067</th>\n",
       "      <td>108067</td>\n",
       "      <td>2417991</td>\n",
       "      <td>2417991</td>\n",
       "      <td>560819</td>\n",
       "      <td>9650692891</td>\n",
       "      <td>VG_100K_2/2417991.jpg</td>\n",
       "      <td>406</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108068</th>\n",
       "      <td>108068</td>\n",
       "      <td>2417992</td>\n",
       "      <td>2417992</td>\n",
       "      <td>41116</td>\n",
       "      <td>9669763691</td>\n",
       "      <td>VG_100K_2/2417992.jpg</td>\n",
       "      <td>372</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108069</th>\n",
       "      <td>108069</td>\n",
       "      <td>2417993</td>\n",
       "      <td>2417993</td>\n",
       "      <td>109761</td>\n",
       "      <td>9680535991</td>\n",
       "      <td>VG_100K_2/2417993.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108070</th>\n",
       "      <td>108070</td>\n",
       "      <td>2417994</td>\n",
       "      <td>2417994</td>\n",
       "      <td>27438</td>\n",
       "      <td>9684712191</td>\n",
       "      <td>VG_100K_2/2417994.jpg</td>\n",
       "      <td>246</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51208 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        meta_dbidx path_vgids meta_vgids meta_cocoids meta_flickr_ids  \\\n",
       "4996          4996    2415074    2415074        33554       128827281   \n",
       "4997          4997    2415075    2415075       290942       163614481   \n",
       "4999          4999    2415077    2415077       111842       177587881   \n",
       "5002          5002    2415080    2415080       100318       393790581   \n",
       "5004          5004    2415082    2415082       127751       452973481   \n",
       "...            ...        ...        ...          ...             ...   \n",
       "108064      108064    2417988    2417988        15751      9419515091   \n",
       "108067      108067    2417991    2417991       560819      9650692891   \n",
       "108068      108068    2417992    2417992        41116      9669763691   \n",
       "108069      108069    2417993    2417993       109761      9680535991   \n",
       "108070      108070    2417994    2417994        27438      9684712191   \n",
       "\n",
       "                   meta_paths  meta_heights  meta_widths  vg150_split  \\\n",
       "4996    VG_100K_2/2415074.jpg           375          500            0   \n",
       "4997    VG_100K_2/2415075.jpg           375          500            0   \n",
       "4999    VG_100K_2/2415077.jpg           375          500            0   \n",
       "5002    VG_100K_2/2415080.jpg           426          500            0   \n",
       "5004    VG_100K_2/2415082.jpg           375          500            0   \n",
       "...                       ...           ...          ...          ...   \n",
       "108064  VG_100K_2/2417988.jpg           333          500            2   \n",
       "108067  VG_100K_2/2417991.jpg           406          500            2   \n",
       "108068  VG_100K_2/2417992.jpg           372          500            2   \n",
       "108069  VG_100K_2/2417993.jpg           500          500            2   \n",
       "108070  VG_100K_2/2417994.jpg           246          500            2   \n",
       "\n",
       "        vgcoco_split  \n",
       "4996               0  \n",
       "4997               0  \n",
       "4999               0  \n",
       "5002               0  \n",
       "5004               0  \n",
       "...              ...  \n",
       "108064             2  \n",
       "108067             2  \n",
       "108068             2  \n",
       "108069             2  \n",
       "108070             2  \n",
       "\n",
       "[51208 rows x 10 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the VGCOCO images, 51,208\n",
    "vgcoco_sets = meta_infos.loc[(np.array(vgcoco_split)>-1)]  # 51,208 images\n",
    "vgcoco_sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_dbidx</th>\n",
       "      <th>path_vgids</th>\n",
       "      <th>meta_vgids</th>\n",
       "      <th>meta_cocoids</th>\n",
       "      <th>meta_flickr_ids</th>\n",
       "      <th>meta_paths</th>\n",
       "      <th>meta_heights</th>\n",
       "      <th>meta_widths</th>\n",
       "      <th>vg150_split</th>\n",
       "      <th>vgcoco_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7224</th>\n",
       "      <td>7224</td>\n",
       "      <td>713568</td>\n",
       "      <td>713568</td>\n",
       "      <td>390520</td>\n",
       "      <td>5996311408</td>\n",
       "      <td>VG_100K/713568.jpg</td>\n",
       "      <td>603</td>\n",
       "      <td>1024</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23165</th>\n",
       "      <td>23165</td>\n",
       "      <td>2398666</td>\n",
       "      <td>2398666</td>\n",
       "      <td>260977</td>\n",
       "      <td>4272927117</td>\n",
       "      <td>VG_100K_2/2398666.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32542</th>\n",
       "      <td>32542</td>\n",
       "      <td>2388883</td>\n",
       "      <td>2388883</td>\n",
       "      <td>11223</td>\n",
       "      <td>329037602</td>\n",
       "      <td>VG_100K_2/2388883.jpg</td>\n",
       "      <td>371</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32942</th>\n",
       "      <td>32942</td>\n",
       "      <td>2388466</td>\n",
       "      <td>2388466</td>\n",
       "      <td>476113</td>\n",
       "      <td>2046220199</td>\n",
       "      <td>VG_100K_2/2388466.jpg</td>\n",
       "      <td>332</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35992</th>\n",
       "      <td>35992</td>\n",
       "      <td>2385261</td>\n",
       "      <td>2385261</td>\n",
       "      <td>362351</td>\n",
       "      <td>6223746372</td>\n",
       "      <td>VG_100K_2/2385261.jpg</td>\n",
       "      <td>310</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38425</th>\n",
       "      <td>38425</td>\n",
       "      <td>2382722</td>\n",
       "      <td>2382722</td>\n",
       "      <td>531831</td>\n",
       "      <td>5342815955</td>\n",
       "      <td>VG_100K_2/2382722.jpg</td>\n",
       "      <td>332</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61458</th>\n",
       "      <td>61458</td>\n",
       "      <td>2358603</td>\n",
       "      <td>2358603</td>\n",
       "      <td>465211</td>\n",
       "      <td>2686767442</td>\n",
       "      <td>VG_100K/2358603.jpg</td>\n",
       "      <td>281</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62781</th>\n",
       "      <td>62781</td>\n",
       "      <td>2357213</td>\n",
       "      <td>2357213</td>\n",
       "      <td>330652</td>\n",
       "      <td>7612513828</td>\n",
       "      <td>VG_100K/2357213.jpg</td>\n",
       "      <td>374</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64852</th>\n",
       "      <td>64852</td>\n",
       "      <td>2355044</td>\n",
       "      <td>2355044</td>\n",
       "      <td>515820</td>\n",
       "      <td>6413178765</td>\n",
       "      <td>VG_100K/2355044.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72722</th>\n",
       "      <td>72722</td>\n",
       "      <td>2346805</td>\n",
       "      <td>2346805</td>\n",
       "      <td>100407</td>\n",
       "      <td>4374150810</td>\n",
       "      <td>VG_100K/2346805.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85333</th>\n",
       "      <td>85333</td>\n",
       "      <td>2333591</td>\n",
       "      <td>2333591</td>\n",
       "      <td>132317</td>\n",
       "      <td>3858332771</td>\n",
       "      <td>VG_100K/2333591.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>375</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98937</th>\n",
       "      <td>98937</td>\n",
       "      <td>2319368</td>\n",
       "      <td>2319368</td>\n",
       "      <td>338104</td>\n",
       "      <td>5131840423</td>\n",
       "      <td>VG_100K/2319368.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100339</th>\n",
       "      <td>100339</td>\n",
       "      <td>2317914</td>\n",
       "      <td>2317914</td>\n",
       "      <td>246184</td>\n",
       "      <td>4240573265</td>\n",
       "      <td>VG_100K/2317914.jpg</td>\n",
       "      <td>332</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105450</th>\n",
       "      <td>105450</td>\n",
       "      <td>2415244</td>\n",
       "      <td>2415244</td>\n",
       "      <td>311025</td>\n",
       "      <td>3756515530</td>\n",
       "      <td>VG_100K_2/2415244.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>333</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106504</th>\n",
       "      <td>106504</td>\n",
       "      <td>2416346</td>\n",
       "      <td>2416346</td>\n",
       "      <td>497187</td>\n",
       "      <td>374005089</td>\n",
       "      <td>VG_100K_2/2416346.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107602</th>\n",
       "      <td>107602</td>\n",
       "      <td>2417507</td>\n",
       "      <td>2417507</td>\n",
       "      <td>413120</td>\n",
       "      <td>6902978840</td>\n",
       "      <td>VG_100K_2/2417507.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        meta_dbidx path_vgids meta_vgids meta_cocoids meta_flickr_ids  \\\n",
       "7224          7224     713568     713568       390520      5996311408   \n",
       "23165        23165    2398666    2398666       260977      4272927117   \n",
       "32542        32542    2388883    2388883        11223       329037602   \n",
       "32942        32942    2388466    2388466       476113      2046220199   \n",
       "35992        35992    2385261    2385261       362351      6223746372   \n",
       "38425        38425    2382722    2382722       531831      5342815955   \n",
       "61458        61458    2358603    2358603       465211      2686767442   \n",
       "62781        62781    2357213    2357213       330652      7612513828   \n",
       "64852        64852    2355044    2355044       515820      6413178765   \n",
       "72722        72722    2346805    2346805       100407      4374150810   \n",
       "85333        85333    2333591    2333591       132317      3858332771   \n",
       "98937        98937    2319368    2319368       338104      5131840423   \n",
       "100339      100339    2317914    2317914       246184      4240573265   \n",
       "105450      105450    2415244    2415244       311025      3756515530   \n",
       "106504      106504    2416346    2416346       497187       374005089   \n",
       "107602      107602    2417507    2417507       413120      6902978840   \n",
       "\n",
       "                   meta_paths  meta_heights  meta_widths  vg150_split  \\\n",
       "7224       VG_100K/713568.jpg           603         1024            0   \n",
       "23165   VG_100K_2/2398666.jpg           333          500            0   \n",
       "32542   VG_100K_2/2388883.jpg           371          500            0   \n",
       "32942   VG_100K_2/2388466.jpg           332          500            0   \n",
       "35992   VG_100K_2/2385261.jpg           310          500            0   \n",
       "38425   VG_100K_2/2382722.jpg           332          500            0   \n",
       "61458     VG_100K/2358603.jpg           281          500            0   \n",
       "62781     VG_100K/2357213.jpg           374          500            0   \n",
       "64852     VG_100K/2355044.jpg           500          500            0   \n",
       "72722     VG_100K/2346805.jpg           333          500            0   \n",
       "85333     VG_100K/2333591.jpg           500          375            2   \n",
       "98937     VG_100K/2319368.jpg           375          500            2   \n",
       "100339    VG_100K/2317914.jpg           332          500            2   \n",
       "105450  VG_100K_2/2415244.jpg           500          333            2   \n",
       "106504  VG_100K_2/2416346.jpg           333          500            2   \n",
       "107602  VG_100K_2/2417507.jpg           333          500            2   \n",
       "\n",
       "        vgcoco_split  \n",
       "7224               0  \n",
       "23165              0  \n",
       "32542              0  \n",
       "32942              0  \n",
       "35992              0  \n",
       "38425              0  \n",
       "61458              0  \n",
       "62781              0  \n",
       "64852              0  \n",
       "72722              0  \n",
       "85333              2  \n",
       "98937              2  \n",
       "100339             2  \n",
       "105450             2  \n",
       "106504             2  \n",
       "107602             2  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We found that there are 16 images that not in coco_entities, so we remove them \n",
    "no_entities_cids = []\n",
    "no_entities_idxes = []\n",
    "for cid, dbidx in zip(vgcoco_sets['meta_cocoids'], vgcoco_sets['meta_dbidx']):\n",
    "    if cid not in coco_entities:\n",
    "        no_entities_cids.append(cid)\n",
    "        no_entities_idxes.append(dbidx)\n",
    "vgcoco_sets.loc[vgcoco_sets['meta_cocoids'].isin(no_entities_cids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51192"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vgcoco_entity_split=vgcoco_split.copy()\n",
    "for idx in no_entities_idxes:\n",
    "    vgcoco_entity_split[idx] = -1\n",
    "len(np.where(np.array(vgcoco_entity_split)>-1)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_dbidx</th>\n",
       "      <th>path_vgids</th>\n",
       "      <th>meta_vgids</th>\n",
       "      <th>meta_cocoids</th>\n",
       "      <th>meta_flickr_ids</th>\n",
       "      <th>meta_paths</th>\n",
       "      <th>meta_heights</th>\n",
       "      <th>meta_widths</th>\n",
       "      <th>vg150_split</th>\n",
       "      <th>vgcoco_split</th>\n",
       "      <th>vgcoco_entity_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K_2/1.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/2.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/3.jpg</td>\n",
       "      <td>480</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/4.jpg</td>\n",
       "      <td>480</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>VG_100K/5.jpg</td>\n",
       "      <td>600</td>\n",
       "      <td>800</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108068</th>\n",
       "      <td>108068</td>\n",
       "      <td>2417992</td>\n",
       "      <td>2417992</td>\n",
       "      <td>41116</td>\n",
       "      <td>9669763691</td>\n",
       "      <td>VG_100K_2/2417992.jpg</td>\n",
       "      <td>372</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108069</th>\n",
       "      <td>108069</td>\n",
       "      <td>2417993</td>\n",
       "      <td>2417993</td>\n",
       "      <td>109761</td>\n",
       "      <td>9680535991</td>\n",
       "      <td>VG_100K_2/2417993.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108070</th>\n",
       "      <td>108070</td>\n",
       "      <td>2417994</td>\n",
       "      <td>2417994</td>\n",
       "      <td>27438</td>\n",
       "      <td>9684712191</td>\n",
       "      <td>VG_100K_2/2417994.jpg</td>\n",
       "      <td>246</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108071</th>\n",
       "      <td>108071</td>\n",
       "      <td>2417995</td>\n",
       "      <td>2417995</td>\n",
       "      <td>None</td>\n",
       "      <td>9721821291</td>\n",
       "      <td>VG_100K_2/2417995.jpg</td>\n",
       "      <td>294</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108072</th>\n",
       "      <td>108072</td>\n",
       "      <td>2417997</td>\n",
       "      <td>2417997</td>\n",
       "      <td>None</td>\n",
       "      <td>9777145091</td>\n",
       "      <td>VG_100K_2/2417997.jpg</td>\n",
       "      <td>332</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108073 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        meta_dbidx path_vgids meta_vgids meta_cocoids meta_flickr_ids  \\\n",
       "0                0          1          1         None            None   \n",
       "1                1          2          2         None            None   \n",
       "2                2          3          3         None            None   \n",
       "3                3          4          4         None            None   \n",
       "4                4          5          5         None            None   \n",
       "...            ...        ...        ...          ...             ...   \n",
       "108068      108068    2417992    2417992        41116      9669763691   \n",
       "108069      108069    2417993    2417993       109761      9680535991   \n",
       "108070      108070    2417994    2417994        27438      9684712191   \n",
       "108071      108071    2417995    2417995         None      9721821291   \n",
       "108072      108072    2417997    2417997         None      9777145091   \n",
       "\n",
       "                   meta_paths  meta_heights  meta_widths  vg150_split  \\\n",
       "0             VG_100K_2/1.jpg           600          800            0   \n",
       "1               VG_100K/2.jpg           600          800            0   \n",
       "2               VG_100K/3.jpg           480          640            0   \n",
       "3               VG_100K/4.jpg           480          640            0   \n",
       "4               VG_100K/5.jpg           600          800            0   \n",
       "...                       ...           ...          ...          ...   \n",
       "108068  VG_100K_2/2417992.jpg           372          500            2   \n",
       "108069  VG_100K_2/2417993.jpg           500          500            2   \n",
       "108070  VG_100K_2/2417994.jpg           246          500            2   \n",
       "108071  VG_100K_2/2417995.jpg           294          500            2   \n",
       "108072  VG_100K_2/2417997.jpg           332          500            2   \n",
       "\n",
       "        vgcoco_split  vgcoco_entity_split  \n",
       "0                 -1                   -1  \n",
       "1                 -1                   -1  \n",
       "2                 -1                   -1  \n",
       "3                 -1                   -1  \n",
       "4                 -1                   -1  \n",
       "...              ...                  ...  \n",
       "108068             2                    2  \n",
       "108069             2                    2  \n",
       "108070             2                    2  \n",
       "108071            -1                   -1  \n",
       "108072            -1                   -1  \n",
       "\n",
       "[108073 rows x 11 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_infos['vgcoco_entity_split'] = vgcoco_entity_split\n",
    "meta_infos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_dbidx</th>\n",
       "      <th>path_vgids</th>\n",
       "      <th>meta_vgids</th>\n",
       "      <th>meta_cocoids</th>\n",
       "      <th>meta_flickr_ids</th>\n",
       "      <th>meta_paths</th>\n",
       "      <th>meta_heights</th>\n",
       "      <th>meta_widths</th>\n",
       "      <th>vg150_split</th>\n",
       "      <th>vgcoco_split</th>\n",
       "      <th>vgcoco_entity_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>4996</td>\n",
       "      <td>2415074</td>\n",
       "      <td>2415074</td>\n",
       "      <td>33554</td>\n",
       "      <td>128827281</td>\n",
       "      <td>VG_100K_2/2415074.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>4997</td>\n",
       "      <td>2415075</td>\n",
       "      <td>2415075</td>\n",
       "      <td>290942</td>\n",
       "      <td>163614481</td>\n",
       "      <td>VG_100K_2/2415075.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>4999</td>\n",
       "      <td>2415077</td>\n",
       "      <td>2415077</td>\n",
       "      <td>111842</td>\n",
       "      <td>177587881</td>\n",
       "      <td>VG_100K_2/2415077.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5002</th>\n",
       "      <td>5002</td>\n",
       "      <td>2415080</td>\n",
       "      <td>2415080</td>\n",
       "      <td>100318</td>\n",
       "      <td>393790581</td>\n",
       "      <td>VG_100K_2/2415080.jpg</td>\n",
       "      <td>426</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5004</th>\n",
       "      <td>5004</td>\n",
       "      <td>2415082</td>\n",
       "      <td>2415082</td>\n",
       "      <td>127751</td>\n",
       "      <td>452973481</td>\n",
       "      <td>VG_100K_2/2415082.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108064</th>\n",
       "      <td>108064</td>\n",
       "      <td>2417988</td>\n",
       "      <td>2417988</td>\n",
       "      <td>15751</td>\n",
       "      <td>9419515091</td>\n",
       "      <td>VG_100K_2/2417988.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108067</th>\n",
       "      <td>108067</td>\n",
       "      <td>2417991</td>\n",
       "      <td>2417991</td>\n",
       "      <td>560819</td>\n",
       "      <td>9650692891</td>\n",
       "      <td>VG_100K_2/2417991.jpg</td>\n",
       "      <td>406</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108068</th>\n",
       "      <td>108068</td>\n",
       "      <td>2417992</td>\n",
       "      <td>2417992</td>\n",
       "      <td>41116</td>\n",
       "      <td>9669763691</td>\n",
       "      <td>VG_100K_2/2417992.jpg</td>\n",
       "      <td>372</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108069</th>\n",
       "      <td>108069</td>\n",
       "      <td>2417993</td>\n",
       "      <td>2417993</td>\n",
       "      <td>109761</td>\n",
       "      <td>9680535991</td>\n",
       "      <td>VG_100K_2/2417993.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108070</th>\n",
       "      <td>108070</td>\n",
       "      <td>2417994</td>\n",
       "      <td>2417994</td>\n",
       "      <td>27438</td>\n",
       "      <td>9684712191</td>\n",
       "      <td>VG_100K_2/2417994.jpg</td>\n",
       "      <td>246</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51192 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        meta_dbidx path_vgids meta_vgids meta_cocoids meta_flickr_ids  \\\n",
       "4996          4996    2415074    2415074        33554       128827281   \n",
       "4997          4997    2415075    2415075       290942       163614481   \n",
       "4999          4999    2415077    2415077       111842       177587881   \n",
       "5002          5002    2415080    2415080       100318       393790581   \n",
       "5004          5004    2415082    2415082       127751       452973481   \n",
       "...            ...        ...        ...          ...             ...   \n",
       "108064      108064    2417988    2417988        15751      9419515091   \n",
       "108067      108067    2417991    2417991       560819      9650692891   \n",
       "108068      108068    2417992    2417992        41116      9669763691   \n",
       "108069      108069    2417993    2417993       109761      9680535991   \n",
       "108070      108070    2417994    2417994        27438      9684712191   \n",
       "\n",
       "                   meta_paths  meta_heights  meta_widths  vg150_split  \\\n",
       "4996    VG_100K_2/2415074.jpg           375          500            0   \n",
       "4997    VG_100K_2/2415075.jpg           375          500            0   \n",
       "4999    VG_100K_2/2415077.jpg           375          500            0   \n",
       "5002    VG_100K_2/2415080.jpg           426          500            0   \n",
       "5004    VG_100K_2/2415082.jpg           375          500            0   \n",
       "...                       ...           ...          ...          ...   \n",
       "108064  VG_100K_2/2417988.jpg           333          500            2   \n",
       "108067  VG_100K_2/2417991.jpg           406          500            2   \n",
       "108068  VG_100K_2/2417992.jpg           372          500            2   \n",
       "108069  VG_100K_2/2417993.jpg           500          500            2   \n",
       "108070  VG_100K_2/2417994.jpg           246          500            2   \n",
       "\n",
       "        vgcoco_split  vgcoco_entity_split  \n",
       "4996               0                    0  \n",
       "4997               0                    0  \n",
       "4999               0                    0  \n",
       "5002               0                    0  \n",
       "5004               0                    0  \n",
       "...              ...                  ...  \n",
       "108064             2                    2  \n",
       "108067             2                    2  \n",
       "108068             2                    2  \n",
       "108069             2                    2  \n",
       "108070             2                    2  \n",
       "\n",
       "[51192 rows x 11 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vgcoco_entity_sets = meta_infos.loc[(np.array(vgcoco_entity_split)>-1)]  # 51,192 images\n",
    "vgcoco_entity_sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract the sg and grounded captions for 51,192 images\n",
    "cap_without_entity_cnt = 0\n",
    "cap_total_cnt = 0\n",
    "cap_sgentities_vgcoco = {}\n",
    "for vid, cid, path, dbidx in zip(vgcoco_entity_sets['meta_vgids'], vgcoco_entity_sets['meta_cocoids'], vgcoco_entity_sets['meta_paths'],\n",
    "                                vgcoco_entity_sets['meta_dbidx']):\n",
    "    all_sgs = capsg[vid]\n",
    "    all_entities = coco_entities[cid]\n",
    "    cap_total_cnt += len(all_sgs)\n",
    "    for sg in all_sgs:\n",
    "        #print(sg)\n",
    "        cap = sentence_preprocess(sg['caption'])\n",
    "        sg['caption'] = cap\n",
    "        if cap in all_entities:\n",
    "            sg['entities'] = all_entities[cap]\n",
    "        else:\n",
    "            cap_without_entity_cnt += 1\n",
    "            sg['entities'] = None\n",
    "    cap_sgentities_vgcoco[vid] = {'coco_id':cid, 'path': path, 'dbidx': dbidx, 'sg': all_sgs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(256103, 8564)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cap_total_cnt, cap_without_entity_cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'coco_id': '33554',\n",
       " 'path': 'VG_100K_2/2415074.jpg',\n",
       " 'dbidx': 4996,\n",
       " 'sg': [{'node': ['tourist-1', \"tourist-1'\", 'sign-5', \"sign-5'\"],\n",
       "   'edge': [['tourist-1', 'find', 'sign-5'],\n",
       "    [\"tourist-1'\", 'find', \"sign-5'\"]],\n",
       "   'caption': 'tourists will find blue signs like this in great britain',\n",
       "   'entities': {'det_sequences': ['_',\n",
       "     None,\n",
       "     None,\n",
       "     'sign',\n",
       "     'sign',\n",
       "     None,\n",
       "     None,\n",
       "     None,\n",
       "     '_',\n",
       "     '_'],\n",
       "    'noun_chunks': [['tourists', '_'],\n",
       "     ['blue signs', 'sign'],\n",
       "     ['great britain', '_']],\n",
       "    'detections': {'sign': [[1,\n",
       "       [124.60584259033203,\n",
       "        8.291168212890625,\n",
       "        449.1304626464844,\n",
       "        303.53790283203125]],\n",
       "      [2,\n",
       "       [177.33352661132812,\n",
       "        164.6728515625,\n",
       "        450.95135498046875,\n",
       "        259.86029052734375]],\n",
       "      [9,\n",
       "       [162.57180786132812,\n",
       "        38.54546356201172,\n",
       "        482.7727355957031,\n",
       "        139.89659118652344]],\n",
       "      [16,\n",
       "       [396.06201171875,\n",
       "        318.5375061035156,\n",
       "        453.98590087890625,\n",
       "        348.01885986328125]],\n",
       "      [28,\n",
       "       [160.9239044189453,\n",
       "        101.58634185791016,\n",
       "        482.4241638183594,\n",
       "        200.08192443847656]]]},\n",
       "    'split': 'train'}},\n",
       "  {'node': ['sign-3', 'connection-10'],\n",
       "   'edge': [['sign-3', 'show from', 'connection-10']],\n",
       "   'caption': 'the road sign shows that this too far from connections',\n",
       "   'entities': {'det_sequences': ['sign',\n",
       "     'sign',\n",
       "     'sign',\n",
       "     None,\n",
       "     None,\n",
       "     None,\n",
       "     None,\n",
       "     None,\n",
       "     None,\n",
       "     '_'],\n",
       "    'noun_chunks': [['the road sign', 'sign'], ['connections', '_']],\n",
       "    'detections': {'sign': [[1,\n",
       "       [124.60584259033203,\n",
       "        8.291168212890625,\n",
       "        449.1304626464844,\n",
       "        303.53790283203125]],\n",
       "      [2,\n",
       "       [177.33352661132812,\n",
       "        164.6728515625,\n",
       "        450.95135498046875,\n",
       "        259.86029052734375]],\n",
       "      [9,\n",
       "       [162.57180786132812,\n",
       "        38.54546356201172,\n",
       "        482.7727355957031,\n",
       "        139.89659118652344]],\n",
       "      [16,\n",
       "       [396.06201171875,\n",
       "        318.5375061035156,\n",
       "        453.98590087890625,\n",
       "        348.01885986328125]],\n",
       "      [28,\n",
       "       [160.9239044189453,\n",
       "        101.58634185791016,\n",
       "        482.4241638183594,\n",
       "        200.08192443847656]]]},\n",
       "    'split': 'train'}},\n",
       "  {'node': ['sign-3', 'direction-5', 'town-8', \"town-8'\", \"town-8''\"],\n",
       "   'edge': [['sign-3', 'give', 'direction-5'],\n",
       "    ['sign-3', 'give to', 'town-8'],\n",
       "    ['sign-3', 'give to', \"town-8'\"],\n",
       "    ['sign-3', 'give to', \"town-8''\"]],\n",
       "   'caption': 'a blue sign giving directions to three towns',\n",
       "   'entities': {'det_sequences': ['sign',\n",
       "     'sign',\n",
       "     'sign',\n",
       "     None,\n",
       "     '_',\n",
       "     None,\n",
       "     'buildings',\n",
       "     'buildings'],\n",
       "    'noun_chunks': [['a blue sign', 'sign'],\n",
       "     ['directions', '_'],\n",
       "     ['three towns', 'buildings']],\n",
       "    'detections': {'buildings': [[14,\n",
       "       [157.9850311279297, 330.2154846191406, 324.0478210449219, 374.375]]],\n",
       "     'sign': [[1,\n",
       "       [124.60584259033203,\n",
       "        8.291168212890625,\n",
       "        449.1304626464844,\n",
       "        303.53790283203125]],\n",
       "      [2,\n",
       "       [177.33352661132812,\n",
       "        164.6728515625,\n",
       "        450.95135498046875,\n",
       "        259.86029052734375]],\n",
       "      [9,\n",
       "       [162.57180786132812,\n",
       "        38.54546356201172,\n",
       "        482.7727355957031,\n",
       "        139.89659118652344]],\n",
       "      [16,\n",
       "       [396.06201171875,\n",
       "        318.5375061035156,\n",
       "        453.98590087890625,\n",
       "        348.01885986328125]],\n",
       "      [28,\n",
       "       [160.9239044189453,\n",
       "        101.58634185791016,\n",
       "        482.4241638183594,\n",
       "        200.08192443847656]]]},\n",
       "    'split': 'train'}},\n",
       "  {'node': ['sogn-7', 'building-9', 'background-12'],\n",
       "   'edge': [['sogn-7', 'with', 'building-9'],\n",
       "    ['building-9', 'in', 'background-12']],\n",
       "   'caption': 'a close up of a street sogn with buildings in the background',\n",
       "   'entities': {'det_sequences': [None,\n",
       "     None,\n",
       "     None,\n",
       "     None,\n",
       "     '_',\n",
       "     '_',\n",
       "     '_',\n",
       "     None,\n",
       "     'buildings',\n",
       "     None,\n",
       "     '_',\n",
       "     '_'],\n",
       "    'noun_chunks': [['a street sogn', '_'],\n",
       "     ['buildings', 'buildings'],\n",
       "     ['the background', '_']],\n",
       "    'detections': {'buildings': [[14,\n",
       "       [157.9850311279297, 330.2154846191406, 324.0478210449219, 374.375]]]},\n",
       "    'split': 'train'}},\n",
       "  {'node': ['people-4', 'traffic-6', 'direction-10'],\n",
       "   'edge': [['people-4', 'in', 'traffic-6'],\n",
       "    ['traffic-6', 'in', 'direction-10']],\n",
       "   'caption': 'street signs point people in traffic in the right direction',\n",
       "   'entities': {'det_sequences': ['sign',\n",
       "     'sign',\n",
       "     None,\n",
       "     '_',\n",
       "     None,\n",
       "     '_',\n",
       "     None,\n",
       "     '_',\n",
       "     '_',\n",
       "     '_'],\n",
       "    'noun_chunks': [['street signs', 'sign'],\n",
       "     ['people', '_'],\n",
       "     ['traffic', '_'],\n",
       "     ['the right direction', '_']],\n",
       "    'detections': {'sign': [[1,\n",
       "       [124.60584259033203,\n",
       "        8.291168212890625,\n",
       "        449.1304626464844,\n",
       "        303.53790283203125]],\n",
       "      [2,\n",
       "       [177.33352661132812,\n",
       "        164.6728515625,\n",
       "        450.95135498046875,\n",
       "        259.86029052734375]],\n",
       "      [9,\n",
       "       [162.57180786132812,\n",
       "        38.54546356201172,\n",
       "        482.7727355957031,\n",
       "        139.89659118652344]],\n",
       "      [16,\n",
       "       [396.06201171875,\n",
       "        318.5375061035156,\n",
       "        453.98590087890625,\n",
       "        348.01885986328125]],\n",
       "      [28,\n",
       "       [160.9239044189453,\n",
       "        101.58634185791016,\n",
       "        482.4241638183594,\n",
       "        200.08192443847656]]]},\n",
       "    'split': 'train'}}]}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cap_sgentities_vgcoco['2415074']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to file\n",
    "meta_infos.to_csv(meta_form_file, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mmcv.dump(cap_sgentities_vgcoco, cap_sgentities_vgcoco_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some statistical results given here\n",
    "1. COCO and VG have **51,208** overlapped images (`vgcoco_split` in `meta_infos`).\n",
    "2. When concating with the `coco_entities` data, we find that 16 images do not have grounded entities. So we removed them, resulting in **51,192** images (`vgcoco_entity_split` in meta_infos)\n",
    "3. Some images have 6 captions. So the 51,192 images have **256,103** image captions. We try our best to post-process our captions to match with `coco_entities` (modify the `sentence_preprocess` function) and **8,564** captions are not found in `coco_entities`. **8,452** of them are not found because `coco_entities` has less than 5 captions. **112** of them are not found because they do not match with any one of the 5 captions. \n",
    "\n",
    "#### Now, we get the most useful data: \n",
    "1. `meta_infos`: it contains the most integrated information of each image.\n",
    "2. `cap_sgentities_vgcoco`:\n",
    "\n",
    "```\n",
    "JSON Format:\n",
    "{\n",
    "\"XXXXX\"(str, vg_id):{\n",
    "    \"coco_id\": str,\n",
    "    \"path\": str('VG_100K_2/2415074.jpg'),\n",
    "    \"dbidx\": int(4996),\n",
    "    \"sg\": [\n",
    "        \"node\": [str],\n",
    "        \"edge\": [[str]],\n",
    "        \"caption\": str,\n",
    "        \"entities\":None, or {\n",
    "            \"det_sequences\": [str],\n",
    "            \"noun_chunks\": [[str]],\n",
    "            \"detections\": {\n",
    "                str: [\n",
    "                [int, [int]]\n",
    "                ]\n",
    "            }\n",
    "        },\n",
    "        \"split\": str (not use this)\n",
    "    ]\n",
    "}\n",
    "...\n",
    "\t\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py3.8-pt1.4.0] *",
   "language": "python",
   "name": "conda-env-py3.8-pt1.4.0-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
